#!/usr/bin/env python3
import os
import re
import subprocess
import shutil
import time
import glob
from datetime import datetime
import csv
import threading

class DMConfigTester:
    def __init__(self):
        self.original_config = "../init_dm8/dm.ini"
        self.backup_config = "../init_dm8/dm.ini.backup"
        self.test_log = "performance_test.log"
        self.results_csv = "test_results.csv"
        self.config_file = "test_config.txt"
        self.sqllog_config = "../../dmdata_tpcc/DAMENG/sqllog.ini"
        
        self.test_combinations = self.load_test_config()
        self.monitoring_active = False
        self.monitoring_thread = None
        self.resource_stats = None
        
        if not os.path.exists(self.backup_config):
            shutil.copy2(self.original_config, self.backup_config)
    
    def load_test_config(self):
        """从配置文件加载测试参数组合"""
        test_combinations = []
        
        if os.path.exists(self.config_file):
            try:
                with open(self.config_file, 'r', encoding='utf-8') as f:
                    for line in f:
                        line = line.strip()
                        if line and not line.startswith('#'):
                            params = {}
                            parts = line.split('|')
                            for part in parts:
                                if '=' in part:
                                    key, value = part.split('=', 1)
                                    params[key.strip()] = value.strip()
                            if params:
                                test_combinations.append(params)
            except Exception as e:
                print(f"读取配置文件失败: {e}")
        
        if not test_combinations:
            test_combinations = [
                {"COMMIT_BATCH": "10", "COMMIT_BATCH_TIMEOUT": "1"},
                {"COMMIT_BATCH": "30", "COMMIT_BATCH_TIMEOUT": "3"}
            ]
            print("使用默认配置参数")
        
        print(f"加载了 {len(test_combinations)} 个测试配置")
        for i, config in enumerate(test_combinations, 1):
            print(f"配置 {i}: {config}")
        
        return test_combinations
    
    def modify_sqllog_config(self, params):
        """修改sqllog.ini配置文件"""
        if not os.path.exists(self.sqllog_config):
            print(f"警告: sqllog.ini文件不存在: {self.sqllog_config}")
            return False
        
        try:
            with open(self.sqllog_config, 'r', encoding='utf-8') as f:
                content = f.read()
            
            # 设置默认值或从参数中获取
            file_num = params.get("SQL_LOG_FILE_NUM", "500")
            switch_limit = params.get("SQL_LOG_SWITCH_LIMIT", "512")
            exec_time_prec_flag = params.get("SQL_LOG_EXEC_TIME_PREC_FLAG", "1")
            buf_total_size = params.get("SQL_LOG_BUF_TOTAL_SIZE", "10240")
            buf_size = params.get("SQL_LOG_BUF_SIZE", "1024")
            buf_keep_cnt = params.get("SQL_LOG_BUF_KEEP_CNT", "6")
            params_delay = params.get("SQL_LOG_PARAMS_DELAY", "0")
            
            # 替换参数
            content = re.sub(r'FILE_NUM\s*=\s*\d+', f'FILE_NUM = {file_num}', content)
            content = re.sub(r'SWITCH_LIMIT\s*=\s*\d+', f'SWITCH_LIMIT = {switch_limit}', content)
            content = re.sub(r'EXECTIME_PREC_FLAG\s*=\s*\d+', f'EXECTIME_PREC_FLAG = {exec_time_prec_flag}', content)
            content = re.sub(r'BUF_TOTAL_SIZE\s*=\s*\d+', f'BUF_TOTAL_SIZE = {buf_total_size}', content)
            content = re.sub(r'BUF_SIZE\s*=\s*\d+', f'BUF_SIZE = {buf_size}', content)
            content = re.sub(r'BUF_KEEP_CNT\s*=\s*\d+', f'BUF_KEEP_CNT = {buf_keep_cnt}', content)
            content = re.sub(r'PARAMS_DELAY\s*=\s*\d+', f'PARAMS_DELAY = {params_delay}', content)
            
            with open(self.sqllog_config, 'w', encoding='utf-8') as f:
                f.write(content)
            
            print(f"已修改sqllog.ini配置:")
            print(f"  FILE_NUM={file_num}, SWITCH_LIMIT={switch_limit}, EXECTIME_PREC_FLAG={exec_time_prec_flag}")
            print(f"  BUF_TOTAL_SIZE={buf_total_size}, BUF_SIZE={buf_size}")
            print(f"  BUF_KEEP_CNT={buf_keep_cnt}, PARAMS_DELAY={params_delay}")
            return True
            
        except Exception as e:
            print(f"修改sqllog.ini配置失败: {e}")
            return False
    
    def generate_config(self, params):
        """生成指定参数的配置文件"""
        print(f"生成配置: {params}")
        
        with open(self.backup_config, 'r', encoding='utf-8') as f:
            content = f.read()
        
        for param_name, param_value in params.items():
            content = self.replace_parameter(content, param_name, param_value)
        
        with open(self.original_config, 'w', encoding='utf-8') as f:
            f.write(content)
        
        return True
    
    def replace_parameter(self, content, param_name, new_value):
        """替换配置文件中的参数值"""
        lines = content.split('\n')
        new_lines = []
        param_found = False
        
        for line in lines:
            if line.strip().startswith(f"{param_name}="):
                new_line = f"{param_name}={new_value}"
                new_lines.append(new_line)
                param_found = True
            else:
                new_lines.append(line)
        
        if not param_found:
            new_lines.append(f"{param_name}={new_value}")
        
        return '\n'.join(new_lines)
    
    def filter_outliers(self, data):
        """过滤掉头尾小于平均值的采样点"""
        if not data or len(data) < 3:
            return data
        
        avg = sum(data) / len(data)
        
        # 从头开始过滤
        start_index = 0
        for i in range(len(data)):
            if data[i] >= avg:
                start_index = i
                break
        
        # 从尾开始过滤
        end_index = len(data) - 1
        for i in range(len(data) - 1, -1, -1):
            if data[i] >= avg:
                end_index = i
                break
        
        return data[start_index:end_index + 1]
    
    def calculate_stats(self, data):
        """计算统计信息"""
        if not data:
            return {"min": 0, "max": 0, "avg": 0, "values": []}
        
        filtered_data = self.filter_outliers(data)
        
        if not filtered_data:
            return {"min": 0, "max": 0, "avg": 0, "values": []}
        
        min_val = min(filtered_data)
        max_val = max(filtered_data)
        avg_val = sum(filtered_data) / len(filtered_data)
        
        return {
            "min": min_val,
            "max": max_val,
            "avg": avg_val,
            "values": filtered_data
        }
    
    def monitor_resources_thread(self, interval=2):
        """资源监控线程函数"""
        cpu_percentages = []
        disk_read_rates = []  # MB/s
        disk_write_rates = []  # MB/s
        sample_count = 0
        
        print("资源监控线程启动...")
        
        # 获取初始磁盘IO计数
        try:
            disk_io_start = self.get_disk_io_counters()
        except:
            disk_io_start = None
        
        while self.monitoring_active:
            try:
                sample_count += 1
                
                # 获取CPU使用率
                cpu_cmd = "top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d'%' -f1"
                cpu_result = subprocess.run(cpu_cmd, shell=True, stdout=subprocess.PIPE, text=True)
                if cpu_result.returncode == 0:
                    cpu_percent = float(cpu_result.stdout.strip())
                    cpu_percentages.append(cpu_percent)
                else:
                    cpu_percentages.append(0)
                
                # 获取磁盘IO速率
                disk_io_current = self.get_disk_io_counters()
                if disk_io_start and disk_io_current:
                    # 转换为MB/s
                    read_rate = (disk_io_current['read_bytes'] - disk_io_start['read_bytes']) / interval / (1024 * 1024)
                    write_rate = (disk_io_current['write_bytes'] - disk_io_start['write_bytes']) / interval / (1024 * 1024)
                    disk_read_rates.append(read_rate)
                    disk_write_rates.append(write_rate)
                    disk_io_start = disk_io_current
                else:
                    disk_read_rates.append(0)
                    disk_write_rates.append(0)
                
                time.sleep(interval)
                
            except Exception as e:
                print(f"资源监控错误: {e}")
                break
        
        # 计算统计信息
        cpu_stats = self.calculate_stats(cpu_percentages)
        disk_read_stats = self.calculate_stats(disk_read_rates)
        disk_write_stats = self.calculate_stats(disk_write_rates)
        
        self.resource_stats = {
            "cpu": cpu_stats,
            "disk_read": disk_read_stats,
            "disk_write": disk_write_stats,
            "interval": interval,
            "samples": sample_count,
            "duration": sample_count * interval,
            "raw_cpu": cpu_percentages,
            "raw_disk_read": disk_read_rates,
            "raw_disk_write": disk_write_rates
        }
        print(f"资源监控完成，收集了 {sample_count} 个样本")
    
    def get_disk_io_counters(self):
        """获取磁盘IO计数器"""
        try:
            # 读取/proc/diskstats获取磁盘IO信息
            with open('/proc/diskstats', 'r') as f:
                lines = f.readlines()
            
            total_read = 0
            total_write = 0
            
            for line in lines:
                parts = line.split()
                if len(parts) >= 14:
                    # 第6个字段是读取的扇区数，第10个字段是写入的扇区数
                    # 1扇区 = 512字节
                    read_sectors = int(parts[5])
                    write_sectors = int(parts[9])
                    total_read += read_sectors * 512  # 转换为字节
                    total_write += write_sectors * 512  # 转换为字节
            
            return {'read_bytes': total_read, 'write_bytes': total_write}
        except:
            return None
    
    def start_monitoring(self):
        """开始资源监控"""
        self.monitoring_active = True
        self.resource_stats = None
        self.monitoring_thread = threading.Thread(target=self.monitor_resources_thread, args=(2,))
        self.monitoring_thread.daemon = True
        self.monitoring_thread.start()
        print("资源监控已启动")
    
    def stop_monitoring(self):
        """停止资源监控"""
        if self.monitoring_active:
            self.monitoring_active = False
            if self.monitoring_thread and self.monitoring_thread.is_alive():
                self.monitoring_thread.join(timeout=3)
                print("资源监控已停止")
            return self.resource_stats
        return None
    
    def format_duration(self, seconds):
        """格式化时间间隔为易读的格式"""
        if seconds < 60:
            return f"{seconds:.1f}秒"
        elif seconds < 3600:
            minutes = seconds // 60
            seconds = seconds % 60
            return f"{minutes}分{seconds:.0f}秒"
        else:
            hours = seconds // 3600
            minutes = (seconds % 3600) // 60
            seconds = seconds % 60
            return f"{hours}时{minutes}分{seconds:.0f}秒"
    
    def run_command_with_timestamp(self, cmd, is_tpcc_test=False):
        """执行命令并带时间戳输出"""
        start_time = time.time()
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        print(f"[{timestamp}] 执行命令: {cmd}")
        
        try:
            if is_tpcc_test:
                # 对于tpcc_test命令，使用Popen以便在运行时监控
                process = subprocess.Popen(cmd, shell=True, 
                                         stdout=subprocess.PIPE, 
                                         stderr=subprocess.PIPE,
                                         text=True)
                
                # 等待5秒后开始监控
                time.sleep(5)
                self.start_monitoring()
                
                # 等待进程结束
                stdout, stderr = process.communicate()
                
                # 命令结束后立即停止监控
                resource_stats = self.stop_monitoring()
                
                if process.returncode == 0:
                    end_time = time.time()
                    duration = end_time - start_time
                    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                    formatted_duration = self.format_duration(duration)
                    print(f"[{timestamp}] 命令执行成功: {cmd} (耗时: {formatted_duration})")
                    return True
                else:
                    raise subprocess.CalledProcessError(process.returncode, cmd, stdout, stderr)
                    
            else:
                # 对于其他命令，使用原来的方式
                result = subprocess.run(cmd, shell=True, check=True, 
                                      stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                      text=True, timeout=3600)
                
                end_time = time.time()
                duration = end_time - start_time
                timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                formatted_duration = self.format_duration(duration)
                print(f"[{timestamp}] 命令执行成功: {cmd} (耗时: {formatted_duration})")
                return True
                
        except subprocess.CalledProcessError as e:
            # 如果命令失败，也要停止监控
            if is_tpcc_test:
                self.stop_monitoring()
            end_time = time.time()
            duration = end_time - start_time
            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            formatted_duration = self.format_duration(duration)
            print(f"[{timestamp}] 命令执行失败: {cmd} (耗时: {formatted_duration})")
            print(f"[{timestamp}] 错误输出: {e.stderr}")
            return False
        except subprocess.TimeoutExpired:
            # 如果命令超时，也要停止监控
            if is_tpcc_test:
                self.stop_monitoring()
            end_time = time.time()
            duration = end_time - start_time
            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            formatted_duration = self.format_duration(duration)
            print(f"[{timestamp}] 命令执行超时: {cmd} (耗时: {formatted_duration})")
            return False
    
    def run_test_suite(self, params):
        """运行测试套件"""
        commands = [
            ("sh ../init_dm8/1initDB.sh", False),
            ("sh tpcc_load.sh", False),
            ("sh tpcc_test.sh", True)  # 标记为tpcc_test命令，需要特殊处理
        ]
        
        print("开始执行测试套件...")
        
        for i, (cmd, is_tpcc_test) in enumerate(commands):
            # 在tpcc_load.sh之后，tpcc_test.sh之前修改sqllog配置
            if i == 1:  # tpcc_load.sh执行完成后
                self.modify_sqllog_config(params)
            
            success = self.run_command_with_timestamp(cmd, is_tpcc_test)
            if not success:
                return False
        
        return True
    
    def get_tpcc_result(self):
        """获取TPCC测试结果"""
        try:
            get_latest_log_cmd = 'ls -lort log/01_tpcc/ | tail -n 1 | awk \'{print $NF}\''
            result = subprocess.run(get_latest_log_cmd, shell=True, 
                                  stdout=subprocess.PIPE, text=True)
            
            if result.returncode == 0:
                latest_log = result.stdout.strip()
                log_path = f"log/01_tpcc/{latest_log}"
                
                if os.path.exists(log_path):
                    with open(log_path, 'r', encoding='utf-8') as f:
                        content = f.read()
                    
                    tpmc_pattern = r"Measured tpmC \(NewOrders\)\s*=\s*([\d.]+)"
                    match = re.search(tpmc_pattern, content)
                    
                    if match:
                        tpmc_value = float(match.group(1))
                        return {
                            "tpmc": tpmc_value,
                            "raw_content": ""  # 不再记录原始内容
                        }
                    else:
                        return {"tpmc": 0, "raw_content": ""}
                else:
                    return {"tpmc": 0, "raw_content": ""}
            else:
                return {"tpmc": 0, "raw_content": ""}
                
        except Exception as e:
            return {"tpmc": 0, "raw_content": ""}
    
    def log_test_result(self, params, tpcc_result, resource_stats):
        """记录测试结果"""
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        
        # 记录到详细日志
        with open(self.test_log, 'a', encoding='utf-8') as f:
            f.write(f"\n{'='*80}\n")
            f.write(f"测试时间: {timestamp}\n")
            f.write(f"配置参数: {params}\n")
            f.write(f"TPMC结果: {tpcc_result['tpmc']}\n")
            
            if resource_stats:
                f.write(f"\nCPU使用情况统计 (tpcc_test运行期间):\n")
                f.write(f"  最小值: {resource_stats['cpu']['min']:.2f}%\n")
                f.write(f"  最大值: {resource_stats['cpu']['max']:.2f}%\n")
                f.write(f"  平均值: {resource_stats['cpu']['avg']:.2f}%\n")
                f.write(f"  采样值: {resource_stats['cpu']['values']}\n")
                
                f.write(f"\n磁盘读取速率统计 (MB/s):\n")
                f.write(f"  最小值: {resource_stats['disk_read']['min']:.2f} MB/s\n")
                f.write(f"  最大值: {resource_stats['disk_read']['max']:.2f} MB/s\n")
                f.write(f"  平均值: {resource_stats['disk_read']['avg']:.2f} MB/s\n")
                f.write(f"  采样值: {resource_stats['disk_read']['values']}\n")
                
                f.write(f"\n磁盘写入速率统计 (MB/s):\n")
                f.write(f"  最小值: {resource_stats['disk_write']['min']:.2f} MB/s\n")
                f.write(f"  最大值: {resource_stats['disk_write']['max']:.2f} MB/s\n")
                f.write(f"  平均值: {resource_stats['disk_write']['avg']:.2f} MB/s\n")
                f.write(f"  采样值: {resource_stats['disk_write']['values']}\n")
                
                f.write(f"\n监控时长: {resource_stats['duration']}秒, 采样间隔: {resource_stats['interval']}秒, 样本数: {resource_stats['samples']}\n")
            else:
                f.write(f"\n资源监控数据不可用\n")
        
        # 记录到CSV文件（简化格式）
        csv_header = False
        if not os.path.exists(self.results_csv):
            csv_header = True
        
        with open(self.results_csv, 'a', encoding='utf-8', newline='') as f:
            writer = csv.writer(f)
            
            if csv_header:
                header = ["timestamp", "tpmc"]
                # 添加CPU统计
                header.extend(["cpu_min", "cpu_max", "cpu_avg"])
                # 添加磁盘IO统计
                header.extend(["disk_read_min", "disk_read_max", "disk_read_avg"])
                header.extend(["disk_write_min", "disk_write_max", "disk_write_avg"])
                # 添加所有配置参数（名称和值分开）
                for param_name in params.keys():
                    header.append(param_name)
                writer.writerow(header)
            
            row = [timestamp, tpcc_result['tpmc']]
            
            # 添加CPU统计
            if resource_stats:
                row.extend([
                    f"{resource_stats['cpu']['min']:.2f}",
                    f"{resource_stats['cpu']['max']:.2f}",
                    f"{resource_stats['cpu']['avg']:.2f}"
                ])
                # 添加磁盘IO统计
                row.extend([
                    f"{resource_stats['disk_read']['min']:.2f}",
                    f"{resource_stats['disk_read']['max']:.2f}",
                    f"{resource_stats['disk_read']['avg']:.2f}"
                ])
                row.extend([
                    f"{resource_stats['disk_write']['min']:.2f}",
                    f"{resource_stats['disk_write']['max']:.2f}",
                    f"{resource_stats['disk_write']['avg']:.2f}"
                ])
            else:
                row.extend(["N/A"] * 9)
            
            # 添加配置参数值（按参数名称的顺序）
            for param_name in params.keys():
                row.append(params[param_name])
            
            writer.writerow(row)
    
    def run_all_tests(self):
        """运行所有测试组合"""
        print("开始性能测试...")
        print("=" * 60)
        
        total_tests = len(self.test_combinations)
        current_test = 1
        
        for params in self.test_combinations:
            print(f"\n运行测试 {current_test}/{total_tests}")
            print(f"参数: {params}")
            
            self.generate_config(params)
            test_success = self.run_test_suite(params)
            
            if test_success:
                # 获取TPCC结果
                tpcc_result = self.get_tpcc_result()
                
                # 记录结果（使用最后监控的资源数据）
                self.log_test_result(params, tpcc_result, self.resource_stats)
                
                print(f"TPMC结果: {tpcc_result['tpmc']}")
                if self.resource_stats:
                    print(f"\nCPU使用统计 (tpcc_test期间):")
                    print(f"  最小值: {self.resource_stats['cpu']['min']:.2f}%")
                    print(f"  最大值: {self.resource_stats['cpu']['max']:.2f}%")
                    print(f"  平均值: {self.resource_stats['cpu']['avg']:.2f}%")
                    print(f"  采样值: {self.resource_stats['cpu']['values']}")
                    
                    print(f"\n磁盘读取速率统计 (MB/s):")
                    print(f"  最小值: {self.resource_stats['disk_read']['min']:.2f} MB/s")
                    print(f"  最大值: {self.resource_stats['disk_read']['max']:.2f} MB/s")
                    print(f"  平均值: {self.resource_stats['disk_read']['avg']:.2f} MB/s")
                    print(f"  采样值: {self.resource_stats['disk_read']['values']}")
                    
                    print(f"\n磁盘写入速率统计 (MB/s):")
                    print(f"  最小值: {self.resource_stats['disk_write']['min']:.2f} MB/s")
                    print(f"  最大值: {self.resource_stats['disk_write']['max']:.2f} MB/s")
                    print(f"  平均值: {self.resource_stats['disk_write']['avg']:.2f} MB/s")
                    print(f"  采样值: {self.resource_stats['disk_write']['values']}")
                else:
                    print(f"资源监控数据不可用")
                print(f"测试完成，结果已记录")
            else:
                print(f"测试失败，跳过结果记录")
            
            current_test += 1
            time.sleep(2)
        
        print("\n所有测试完成！")
        print(f"详细日志: {self.test_log}")
        print(f"CSV结果: {self.results_csv}")
    
    def cleanup(self):
        """清理，恢复原始配置"""
        if os.path.exists(self.backup_config):
            shutil.copy2(self.backup_config, self.original_config)
            print("已恢复原始配置文件")

if __name__ == "__main__":
    tester = DMConfigTester()
    
    try:
        tester.run_all_tests()
    except KeyboardInterrupt:
        print("\n测试被用户中断")
        tester.stop_monitoring()
    except Exception as e:
        print(f"测试过程中发生错误: {e}")
        tester.stop_monitoring()
    finally:
        tester.cleanup()
